require 'rubygems'
require 'mechanize'
require 'json'
require 'xml'

STDOUT.sync=true;

codelist = ['ACCT', 'BIEN','BTEC','CBME','CENG','CHEM','CIEM','CIVL','COMP','CSIT',
  'ECON','EEMT','EESM','ELEC','ENEG','ENGG','ENVR','ENVS','EVNG','EVSM',
  'FINA','FYTG','GBUS','GNED','HART','HLTH','HUMA','IBTM','IELM','ISOM',
  'JEVE','LABU','LAGR','LANG','LIFS','MAFS','MALS','MARK','MATH','MECH',
  'MESF','MGMT','NANO','PDEV','PHYS','RMBI','SBMT','SCED','SCIE','SHSS','SOSC','SSMA','TEMG','UROP']

def TheNode(name, content)
  n = XML::Node.new(name)
  n << content
  return n
end

agent = Mechanize.new
count = 0

codelist.each do |cl|

url = 'https://w5.ab.ust.hk/wcq/cgi-bin/1230/subject/'+cl
page = agent.get(url)

allcourses = page.search('[class="course"]')

doc = XML::Document.new
doc.root = XML::Node.new(cl)

allcourses.each do |c|
  courseTag = c.search('h2')[0].text
  courseCode = ((/\w+\s+\w+\s+-/.match(courseTag))[0])[0..-2].strip
  puts courseCode
  courseName = ((/-.+\(/.match(courseTag))[0])[1..-2].strip
  courseUnit = /\(\d/.match(courseTag)[0][1..-1]
  
  course = XML::Node.new('course')
  course['id'] = courseCode
  cName = TheNode('courseName',courseName)
  cUnit = TheNode('courseUnit',courseUnit)
  course<<cName
  course<<cUnit
  
  
  courseContent = c.search('[class="sections"]').search('tr')
  
  cSec = XML::Node.new('section')
  sect = ""
  www = nil
  
  (1..courseContent.length-1).each do |i|
    cc=courseContent[i]
    cc_td = cc.search('td')
    
    isnew = (/newsect/.match(cc.attr('class'))!=nil)
    if(isnew)   
      ofst = 0
      cSec = XML::Node.new('section')
      sect = ((/\w+\s+\(/.match(cc_td[0].text))[0])[0..-2].strip
    else
      ofst = -1
    end
    
    www = XML::Node.new('WWW')
    
    cSec['id'] = sect
    
    datetime = /[Mo|Tu|We|Th|Fr|Sa]+\s\d\d:\d\d\w\w\s-\s\d\d:\d\d\w\w/.match(cc_td[1+ofst].text)
    if(datetime==nil)
      datetime='TBA'
    else
      datetime = datetime[0]
    end
    
    cplace = cc_td[2+ofst].text
    cprof = cc_td[3+ofst]
    
    www << TheNode('dateTime',datetime)
    www << TheNode('place',cplace)
    cProfs = XML::Node.new('profs')
    if(cprof.search('a').length==0)
      cProfs << TheNode('prof',cprof.text)
    else
      cprof.search('a').each do |p|
        cProfs << TheNode('prof',p.text)
      end
    end
    www << cProfs
    cSec << www
    
    if(i!=courseContent.length-1)
      if(/newsect/.match(courseContent[i+1].attr('class'))!=nil)
        course << cSec
        next
      end
    else
    end
    course << cSec
  end
  
  doc.root << course
 end
 
  puts cl+' '+count.to_s
  doc.save(cl+'.xml', :indent=>true, :encoding => XML::Encoding::UTF_8)
  count = count+1
end